package others;

import java.io.*;
import java.util.ArrayList;

/**
 * 严格来说, 这个文件不属于这个项目所代表的模块: 活跃度提取, 但是本文件功能太过简单:
 * 根据已有的noise_word文件和vocabulary文件, 清洗vocabulary和words_stemed文件夹下所有语料, 删掉所有噪声文件
 * 因为需要对语料库中所有文件清洗, 从而生成新的语料库文件, 所以涉及到大量的IO操作,
 * 原先是用Python实现的, 无奈10万级的IO处理太慢了, 只得把这部分代码提出来用Java实现
 * 这就是本文件诞生的原因
 * 因为功能简单, 懒得将它额外开一个新项目, 所以直接插在这里了;
 * 就是这样, 喵**/

public class NewCorpusFilter {

    static ArrayList<String> vocabulary;
    static ArrayList<String> noise_words;

    public static void main(String[] args){
        String root = "/home/wanglinhui/code/WorkSpace_CODE/WorkSpace_Python/2LSTM/data/Bug_msg_eclipse/";
        String vocabulary_path = root + "vocabulary.txt";
        String filtered_vocabulary_path = root + "filtered_vocabulary.txt";
        String noise_words_path = root + "noise_words.txt";
        String old_words_stemed_path = root + "keywords_stemed/";
        String filtered_words_stemed_path = root + "filtered_keywords_stemed/";

        try{
            //将词汇表和噪声文件表写入内存
            vocabulary = readFile2List(vocabulary_path);
            noise_words = readFile2List(noise_words_path);
            //清理词汇表并更新
            cleanVocabularyAndCreateNew(filtered_vocabulary_path);
            //清理语料库并更新
            cleanDocsAndCreateNew(old_words_stemed_path, filtered_words_stemed_path);

        }catch (Exception e){
            e.printStackTrace();
        }
    }
    /**
     * 读取文件, 返回一个列表**/
    public static ArrayList<String> readFile2List(String fileName) throws Exception{
        ArrayList<String> results = new ArrayList<>();
        File file = new File(fileName);
        BufferedReader br = new BufferedReader(new FileReader(file));
        String line = "";
        while ((line = br.readLine()) != null){
            results.add(line.trim());
        }
        br.close();
        return results;
    }
    /**
     * 将列表一个元素一行写入指定文件**/
    public static void writeList2File(String fileName, ArrayList<String> list) throws Exception{
        File file = new File(fileName);
        BufferedWriter bw = new BufferedWriter(new FileWriter(file));
        for (int i=0; i<list.size(); i++){
            bw.write(list.get(i) + "\n");
        }
        bw.close();
    }
    /**
     * 清洗vocabulary, 删掉其中的噪声数据**/
    public static void cleanVocabularyAndCreateNew(String path) throws Exception{
        for (String w: noise_words){
            if (vocabulary.contains(w)){
                vocabulary.remove(w);
            }
        }
        writeList2File(path, vocabulary);


    }
    public static void cleanDocsAndCreateNew(String old_path, String new_path) throws Exception{
        File folder = new File(old_path);
        String[] nameList = folder.list();
        int count = 0;
        for (String name: nameList){
            System.out.println(count++);
            String old_file_path = old_path + name;
            String new_file_path = new_path + name;
            ArrayList<String> words = readFile2List(old_file_path);
            words.removeAll(noise_words);       //移除该文件中的全部噪声数据, removeall可以移除重复值, 甚是方便
            //将清洗之后的words列表写入对应文件
            writeList2File(new_file_path, words);

        }
    }
}
